from scrapy.spiders import CrawlSpider
import scrapy
# from example.items import SeemeispiderItem
from search_news.items import NewsItem


class MySpider(CrawlSpider):
    name = 'navSpider'
    start_urls = ["http://nav.jmu.edu.cn/hydt.htm"]

    def parse(self, response):
        list = response.xpath('//div[@class="artic_t_1 ny_news_lb"]/ul/li')
        print(list)
        for i in list:
            item = NewsItem()
            url = i.xpath(".//a/@href").extract_first()
            item['title'] = i.xpath('.//a/text()').extract_first()
            item['pubdate'] = i.xpath('.//span/text()').extract_first()
            if '../' in url:
                item['url'] = 'http://nav.jmu.edu.cn' + url.replace('../', '')
            else:
                item['url'] = i.xpath(".//a/@href").extract_first()
            item['college'] = "航海学院"

            yield item


        next_page = response.xpath("//a[@class='Next'][1]/@href").extract_first()
        print(next_page)
        if next_page is not None:
            next_page = response.urljoin(next_page)
            yield scrapy.Request(next_page, callback=self.parse, dont_filter=True)